# 按token来切割文档
from langchain.text_splitter import CharacterTextSplitter

#要切割的文档
with open("test.txt", encoding="utf-8") as f:
    zuizhonghuanxiang = f.read()

#初始化切分器
text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
    chunk_size=4000,#切分的文本块大小，一般通过长度函数计算
    chunk_overlap=30,#切分的文本块重叠大小，一般通过长度函数计算
)

text = text_splitter.create_documents([zuizhonghuanxiang])
print(text[0])

